package TAIC.text;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.PrintStream;
import java.util.HashMap;
import java.util.Scanner;

public class VectorizeFiles {
	
	public static void main ( String arg []) throws Exception {
		//System.out.println ( (new Scanner ("")).delimiter() ) ;
		if ( arg.length < 2 ) {
			System.out.println ( "plese input langFile and the corpus to handle") ;
			return ; 
		}
		(new VectorizeFiles ( arg[ 0 ] )).work( arg [ 1 ]) ;
		//processFile ("file453.txt" ) ; //
	} 
	
	public VectorizeFiles ( String langVec ) {
		lang = new LanguageVec ( langVec ) ;
		lang.init() ; 
	}
	
	void work ( String configFile ) {
		//System.setErr( null ) ; 
		try {
			Scanner allFiles = new Scanner ( new FileInputStream ( configFile )) ;
			PrintStream fout = new PrintStream ( new FileOutputStream ( "svmCorpus")) ; 
			while ( allFiles.hasNextInt() ) {
				int classname = allFiles.nextInt() ; 
				HashMap < Integer, Integer > result = processFile ( allFiles.nextLine().trim() ) ;
				if ( result.size() < 2 ) continue ;
				fout.print ( classname ) ; 
				for ( int i = 0; i < lang.size() ; i ++ ) { 
					Integer id ; 
					if ( (id = result.get( i )) != null ) fout.print( " " + i + ":" + id ) ;
				}
				fout.println() ; 
			}
			fout.close (); 
			lang.save() ; 
		}catch ( Exception e ) {
			System.out.println ( "wrong") ;
			e.printStackTrace() ;
		}
		
	}
	
	HashMap < Integer, Integer> processFile ( String filename ) throws Exception {
		Scanner scanner = 
			new Scanner ( new FileInputStream ( filename )).useDelimiter("((<[^>]*>)|(\\p{javaWhitespace}))+");
		HashMap < Integer, Integer > result = new HashMap < Integer, Integer >  () ;
		System.out.println ( filename ) ;
		try{
		//	PrintStream fout = new PrintStream ( new File ( "b.txt")) ;
			while ( scanner.hasNext()  ) {
				String str = scanner.next();
				//if ( str.length() == 0 ) continue ; 
			//	fout.println ( str + " ") ;
				str = LanguageVec.normallize( str ) ;
				if ( ban( str ) ) continue ;
				int id = lang.getID( str ) ;
				Integer count = result.get( id ) ;
				if ( count == null ) count = 0 ;
				result.put( id ,  count + 1 ) ; 
			}
		}catch ( Exception e ) {
		}
		return result;  
	}
	
	boolean ban ( String str ) {
		if ( str.length() < 3  ) return  true ; 
		if ( lang != null && lang.filename.equals( "english.txt") && lang.vec.get( str ) == null ) return true;  
		if ( str.equals( "this" ) || str.equals( "that" ) || str.equals( "the" ) || str.equals( "and" ) ||
				str.equals( "for" ) || str.equals( "you" ) || str.equals( "his" ) || str.equals( "her" ) ||
				str.equals( "these" ) || str.equals( "those" ) || str.equals( "what" ) || str.equals( "which" ) ||
				str.equals( "our" ) || str.equals( "can" ) || str.equals( "will" ) || str.equals( "would" ) ||
				str.equals( "could" ) || str.equals( "may" ) || str.equals( "might" ) || str.equals( "our" ) ||
				str.equals( "here" ) || str.equals( "there" ) || str.equals( "himself" ) || str.equals( "herself" ) ||
				str.equals( "but" ) || str.equals( "how" ) || str.equals( "ourselves" ) || str.equals( "theirselves" ) ||
				str.equals( "myself" ) || str.equals( "yourself" ) || str.equals( "have" ) || str.equals( "has" ) ||
				str.equals( "from" ) || str.equals( "between" ) || str.equals( "beside" ) || str.equals( "behind" ) ||
				str.equals( "when" ) || str.equals( "where" ) || str.equals( "who" ) || str.equals( "are" ) ||
				str.equals( "she" ) || str.equals( "your" ) || str.equals( "yours" ) || str.equals( "with" ) 
		) return true ;
		return false; 
	}
	
	
	LanguageVec lang = null ; 
}
